# -*- coding: utf-8 -*-
import scrapy

import logging
import re
import json
import random
import urllib.parse
from lxml import etree
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem
from FDC_spider.constants import SJZ_STATUS_DICT

logger = logging.getLogger(__name__)


class ShijiazhuangSpider(scrapy.Spider):
    name = 'shijiazhuang'
    allowed_domains = ['sjz.gov.cn']
    start_urls = ['http://zjj.sjz.gov.cn/plus/scxx_zxlp.php?type=1', 'http://zjj.sjz.gov.cn/plus/scxx_zxlp.php?type=2']
    next_page_temp_url = 'http://zjj.sjz.gov.cn/plus/scxx_zxlp.php?pageno={}&type={}'  # 项目列表下一页url
    project_temp_url = 'http://zjj.sjz.gov.cn/plus/scxx_xmdetail.php?id={}'  # 项目详情url
    menu_temp_url = 'http://zjj.sjz.gov.cn/plus/cxda_ys_json_menu.php?id={}'  # 菜单url(包括许可证、楼栋、单元)
    room_li_temp = 'http://zjj.sjz.gov.cn/plus/scxx_subroom_showx.php?{}&numvar={}'  # POST 房号列表url

    def parse(self, response):
        page_li = response.xpath("//div[@id='geren']/table/tr[position()=last()]//div/text()").extract()
        page_str = re.sub(r'\s', '', ''.join(page_li))
        try:
            current_page = re.findall(r'\d+', page_str)[0]
            total_page = re.findall(r'\d+', page_str)[1]
        except Exception as e:
            logger.error('{}  获取分页数错误  error：{}'.format(response.request.url, e))
        else:
            type = '住宅' if response.request.url.find('type=1') > 1 else '非住宅'
            print('>>>>>>>>{}-第{}页<<<<<<<<'.format(type, current_page))
            # 获取项目列表
            tr_li = response.xpath("//div[@id='geren']/table/tr[position()>2 and position()!=last()]")
            if len(tr_li) and len(tr_li) <= 10:
                for tr in tr_li:
                    item_eg = FdcEstateGuidItem()
                    item_eg['roomUse'] = type
                    item_eg['projectName'] = tr.xpath("./td[1]//text()").extract_first()
                    item_eg['developerName'] = tr.xpath("./td[2]/text()").extract_first()
                    item_eg['estateAddress'] = tr.xpath("./td[3]/text()").extract_first()
                    item_eg['salesTel'] = tr.xpath("./td[4]/text()").extract_first()
                    try:
                        projectUrl_str = tr.xpath("./td[1]/a/@href").extract_first()
                        item_eg['projectId'] = re.findall(r'\d+', projectUrl_str)[0]
                        item_eg['projectUrl'] = ShijiazhuangSpider.project_temp_url.format(item_eg['projectId'])
                    except Exception as e:
                        logger.error(
                            '{}  {}-第{}页-{} 项目id获取错误  error:{}'.format(response.request.url, type,
                                                                       item_eg['projectName'],
                                                                       current_page, e))
                    else:
                        yield response.follow(
                            item_eg['projectUrl'],
                            callback=self.parse_project,
                            meta={'item_eg': deepcopy(item_eg)},
                            priority=6,
                        )
            else:
                logger.error('{}  {}-第{}页项目列表获取错误'.format(response.request.url, type, current_page))

            # 翻页
            if int(current_page) < int(total_page):
                next_page_url = ShijiazhuangSpider.next_page_temp_url.format(int(current_page) + 1,
                                                                             1) if type == '住宅' else ShijiazhuangSpider.next_page_temp_url.format(
                    int(current_page) + 1, 2)
                yield scrapy.Request(
                    next_page_url,
                    priority=5,
                )

    def parse_project(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        floorAreaRatio_li = response.xpath("//td[contains(text(),'容')]/following-sibling::td[1]/text()").extract()
        item_eg['floorAreaRatio'] = re.sub(r'\s', '', ''.join(floorAreaRatio_li))
        estateTotalArea_li = response.xpath("//td[contains(text(),'总建筑')]/following-sibling::td[1]/text()").extract()
        estateTotalArea_str = re.sub(r'\s', '', ''.join(estateTotalArea_li))
        item_eg['estateTotalArea'] = re.findall(r'\d+', estateTotalArea_str)[0] if len(
            re.findall(r'\d+', estateTotalArea_str)) else None
        qualificationCertificateNo_li = response.xpath(
            "//td[contains(text(),'资质证书编号')]/following-sibling::td[1]/text()").extract()
        item_eg['qualificationCertificateNo'] = re.sub(r'\s', '', ''.join(qualificationCertificateNo_li))
        qualificationGrade_li = response.xpath(
            "//td[contains(text(),'资质证书等级')]/following-sibling::td[1]/text()").extract()
        item_eg['qualificationGrade'] = re.sub(r'\s', '', ''.join(qualificationGrade_li))
        # 2020/08/21 by ZSQ for 楼盘字典添加新字段
        commencement_date = response.xpath("//td[contains(text(),'施工开始日期')]/following-sibling::td[1]/text()").extract()
        item_eg['commencementDate'] = re.sub(r'\s', '', ''.join(commencement_date))
        completion_date = response.xpath("//td[contains(text(),'施工结束日期')]/following-sibling::td[1]/text()").extract()
        item_eg['completionDate'] = re.sub(r'\s', '', ''.join(completion_date))
        yield scrapy.Request(
            ShijiazhuangSpider.menu_temp_url.format(item_eg['projectId']),
            callback=self.parse_menu,
            meta={'item_eg': deepcopy(item_eg)},
            priority=7,
        )

    def parse_menu(self, response):
        """
        获取信息列表（包括：许可证、楼栋、单元信息）
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        resp_li = json.loads(response.body.decode())
        if len(resp_li):
            for permit_dict in resp_li:
                permit_url = permit_dict['attributes'].get('url', None)
                item_eg['preSalePermit'] = permit_dict.get('text', None)
                if permit_url:
                    yield response.follow(
                        permit_url,
                        callback=self.parse_permit,
                        meta={'item_eg': deepcopy(item_eg)},
                        priority=9,
                    )
                else:
                    logger.warning(
                        '{}  {}-{}-{}  许可证详情url获取失败'.format(response.request.url, item_eg['roomUse'],
                                                            item_eg['projectName'], item_eg['preSalePermit']))
                # 楼栋列表
                bd_li = permit_dict.get('children', [])
                if len(bd_li):
                    for bd_dict in bd_li:
                        item_bd = FdcBuildingItem()
                        item_bd['projectName'] = item_eg['projectName']
                        item_bd['preSalePermit'] = item_eg['preSalePermit']
                        item_bd['projectId'] = item_eg['projectId']
                        item_bd['buildingId'] = bd_dict.get('id', None)
                        item_bd['blockName'] = bd_dict.get('text', None)
                        item_bd['buildingUrl'] = bd_dict['attributes'].get('url', None)
                        #  2020/08/24 by ZSQ for 楼栋字典添加新字段
                        bd_unit_li = list()

                        # 单元列表
                        unit_li = bd_dict.get('children', [])
                        if len(unit_li):
                            for unit_dict in unit_li:
                                item_rm = FdcRoomItem()
                                item_rm['projectName'] = item_bd['projectName']
                                item_rm['buildingId'] = item_bd['buildingId']
                                item_rm['blockName'] = item_bd['blockName']
                                item_rm['unitName'] = unit_dict.get('text', None)
                                unit_url = unit_dict['attributes'].get('url', None)
                                # 获取房号列表
                                bd_unit_li.append(item_rm['unitName']) if item_rm['unitName'] else None
                                room_li_url = self._get_room_li_url(unit_url)
                                if room_li_url:
                                    # 2020/08/24 by ZSQ for 网站改版，房号列表进行加密，改为异步POST请求
                                    # yield response.follow(
                                    #     unit_url,
                                    #     callback=self.parse_room_li,
                                    #     meta={'item_rm': deepcopy(item_rm)},
                                    # )
                                    headers = {
                                        'Referer': urllib.parse.urljoin(response.request.url, unit_url)
                                    }
                                    yield scrapy.Request(
                                        room_li_url,
                                        method='POST',
                                        headers=headers,
                                        callback=self.parse_room_li,
                                        meta={'item_rm': deepcopy(item_rm)},
                                        priority=8,
                                    )
                                else:
                                    logger.warning(
                                        '{}  {}-{}-{}-{}-{}  房号列表url获取失败'.format(response.request.url,
                                                                                 item_eg['roomUse'],
                                                                                 item_eg['projectName'],
                                                                                 item_eg['preSalePermit'],
                                                                                 item_bd['blockName'],
                                                                                 item_rm['unitName']))
                        else:
                            logger.warning(
                                '{}  {}-{}-{}-{}  单元列表获取为空'.format(response.request.url, item_eg['roomUse'],
                                                                   item_eg['projectName'], item_eg['preSalePermit'],
                                                                   item_bd['blockName']))

                        # 获取楼栋详情
                        item_bd['unitLi'] = bd_unit_li if bd_unit_li else None
                        if item_bd['buildingUrl']:
                            yield response.follow(
                                item_bd['buildingUrl'],
                                callback=self.parse_building,
                                meta={'item_bd': deepcopy(item_bd)},
                                priority=9,
                            )
                        else:
                            logger.warning(
                                '{}  {}-{}-{}-{}  楼栋详情url获取失败'.format(response.request.url, item_eg['roomUse'],
                                                                      item_eg['projectName'], item_eg['preSalePermit'],
                                                                      item_bd['blockName']))
                else:
                    logger.warning(
                        '{}  {}-{}-{}  楼栋列表获取为空'.format(response.request.url, item_eg['roomUse'],
                                                        item_eg['projectName'], item_eg['preSalePermit']))
        else:
            logger.warning(
                '{}  {}-{}  许可证列表获取为空'.format(response.request.url, item_eg['roomUse'], item_eg['projectName']))

    def parse_permit(self, response):
        """
        获取预售许可证详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        propertyCompany_li = response.xpath("//td[contains(text(),'物业公司')]/following-sibling::td[1]/text()").extract()
        item_eg['propertyCompany'] = propertyCompany_li[0].strip() if propertyCompany_li[0] and propertyCompany_li[
            0].strip() != '0' else None
        residenceProposedPrice_li = response.xpath(
            "//td[contains(text(),'住宅拟定')]/following-sibling::td[1]/text()").extract()
        item_eg['residenceProposedPrice'] = re.findall(r'\d+', ''.join(residenceProposedPrice_li))[0] if len(
            re.findall(r'\d+', ''.join(residenceProposedPrice_li))) else None
        item_eg['residenceProposedPrice'] = item_eg['residenceProposedPrice'] if item_eg[
                                                                                     'residenceProposedPrice'] != '0' else None
        commercialProposedPrice_li = response.xpath(
            "//td[contains(text(),'商业拟定')]/following-sibling::td[1]/text()").extract()
        item_eg['commercialProposedPrice'] = re.findall(r'\d+', ''.join(commercialProposedPrice_li))[0] if len(
            re.findall(r'\d+', ''.join(commercialProposedPrice_li))) else None
        item_eg['commercialProposedPrice'] = item_eg['commercialProposedPrice'] if item_eg[
                                                                                       'commercialProposedPrice'] != '0' else None
        officialProposedPrice_li = response.xpath(
            "//td[contains(text(),'办公拟定')]/following-sibling::td[1]/text()").extract()
        item_eg['officialProposedPrice'] = re.findall(r'\d+', ''.join(officialProposedPrice_li))[0] if len(
            re.findall(r'\d+', ''.join(officialProposedPrice_li))) else None
        item_eg['officialProposedPrice'] = item_eg['officialProposedPrice'] if item_eg[
                                                                                   'officialProposedPrice'] != '0' else None
        yield item_eg

    def parse_building(self, response):
        """
        获取楼栋详情
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        item_bd['buildingUrl'] = response.request.url
        parkingSpacesNum_li = response.xpath(
            "//td[contains(text(),'停车位')]/following-sibling::td[1]/text()").extract()
        item_bd['parkingSpacesNum'] = ''.join(parkingSpacesNum_li).strip() if len(parkingSpacesNum_li) else None
        item_bd['parkingSpacesNum'] = item_bd['parkingSpacesNum'] if item_bd['parkingSpacesNum'] != '0' else None

        totalRoomNo_li = response.xpath(
            "//td[contains(text(),'总套数')]/following-sibling::td[1]/text()").extract()
        item_bd['totalRoomNo'] = ''.join(totalRoomNo_li).strip() if len(totalRoomNo_li) else None
        item_bd['totalRoomNo'] = item_bd['totalRoomNo'] if item_bd['totalRoomNo'] != '0' else None

        propertyManagementFee_li = response.xpath(
            "//td[contains(text(),'物业管理')]/following-sibling::td[1]/text()").extract()
        item_bd['propertyManagementFee'] = ''.join(propertyManagementFee_li).strip() if len(
            propertyManagementFee_li) else None
        item_bd['propertyManagementFee'] = item_bd['propertyManagementFee'] if item_bd[
                                                                                   'propertyManagementFee'] != '0' else None

        avgPrice_li = response.xpath(
            "//td[contains(text(),'均价')]/following-sibling::td[1]/text()").extract()
        item_bd['avgPrice'] = ''.join(avgPrice_li).strip() if len(avgPrice_li) else None
        item_bd['avgPrice'] = item_bd['avgPrice'] if item_bd['avgPrice'] != '0' else None

        floor_li = response.xpath(
            "//td[contains(text(),'层数')]/following-sibling::td[1]/text()").extract()
        floor_li = re.findall(r'\d+', ''.join(floor_li))
        if len(floor_li):
            item_bd['floorOverGroundNo'] = floor_li[0] if len(floor_li) == 1 else floor_li[1]
            item_bd['floorUnderGroundNo'] = floor_li[0] if len(floor_li) == 2 else None
        else:
            item_bd['floorOverGroundNo'] = None
            item_bd['floorUnderGroundNo'] = None

        # 2020/08/24 by ZSQ for 楼栋字典添加新字段
        commencement_date_li = response.xpath("//td[contains(text(),'开工日期')]/following-sibling::td[1]/text()").extract()
        item_bd['commencementDate'] = self._deal_item_li(commencement_date_li)
        opening_date_li = response.xpath("//td[contains(text(),'开盘时间')]/following-sibling::td[1]/text()").extract()
        item_bd['openingDate'] = self._deal_item_li(opening_date_li)
        handover_date_li = response.xpath("//td[contains(text(),'入住时间')]/following-sibling::td[1]/text()").extract()
        item_bd['handoverDate'] = self._deal_item_li(handover_date_li)
        yield item_bd

    #  2020/08/24 by ZSQ for 网站改版，房号列表进行加密
    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        # # 获取楼层分组
        # tr_li = response.xpath("//table//tr")
        # if len(tr_li):
        #     for tr in tr_li:
        #         item_rm['roomFloor'] = tr.xpath("./td[1]/text()").extract_first()
        #         # 获取当前楼层的房号分组
        #         td_li = tr.xpath("./td[position()>1]")
        #         if len(td_li):
        #             for td in td_li:
        #                 item_rm['roomNo'] = td.xpath("./text()").extract_first()
        #                 status_str = td.xpath("./img/@src").extract_first()
        #                 item_rm['saleStatus'] = SJZ_STATUS_DICT.get(status_str, None)
        #                 yield item_rm
        #         else:
        #             logger.warning(
        #                 '{}  {}-{}-{}-{} 当前楼层房号分组为空'.format(response.request.url, item_rm['projectName'],
        #                                                     item_rm['blockName'], item_rm['unitName'],
        #                                                     item_rm['roomFloor']))
        # else:
        #     logger.warning(
        #         '{}  {}-{}-{} 获取楼层分组为空'.format(response.request.url, item_rm['projectName'], item_rm['blockName'],
        #                                        item_rm['unitName']))
        resp = self._deal_room_li_resp(response)
        if resp:
            resp_html = etree.HTML(resp)
            # 获取楼层分组
            tr_li = resp_html.xpath("//tr")
            if len(tr_li):
                for tr in tr_li:
                    item_rm['roomFloor'] = tr.xpath("./td[1]/text()")[0] if len(tr.xpath("./td[1]/text()")) else None
                    # 获取当前楼层的房号分组
                    td_li = tr.xpath("./td[position()>1]")
                    if len(td_li):
                        for td in td_li:
                            status_str = td.xpath("./img/@src")[0] if len(td.xpath("./img/@src")) else None
                            item_rm['saleStatus'] = SJZ_STATUS_DICT.get(status_str, None)
                            item_rm['roomNo'] = td.xpath("./a/text()")[0] if len(td.xpath("./a/text()")) else None
                            room_url = td.xpath("./a/@href")[0] if len(td.xpath("./a/@href")) else None
                            if room_url:
                                # 获取房号详情
                                headers = {
                                    'Referer': response.request.headers['Referer'].decode()
                                }
                                yield response.follow(
                                    room_url,
                                    headers=headers,
                                    callback=self.parse_room_detail,
                                    meta=dict(item_rm=deepcopy(item_rm), ),
                                    priority=10,
                                )
                            else:
                                logger.warning(
                                    '{}-{}-{}-{}—{} 房号详情url提取为空'.format(item_rm['projectName'], item_rm['blockName'],
                                                                        item_rm['unitName'], item_rm['roomFloor'],
                                                                        item_rm['roomNo']))
                    else:
                        logger.warning('{}-{}-{}-{} 该楼层的房号列表为空'.format(item_rm['projectName'], item_rm['blockName'],
                                                                       item_rm['unitName'], item_rm['roomFloor']))
            else:
                logger.warning(
                    '{}-{}-{} 获取楼层分组为空'.format(item_rm['projectName'], item_rm['blockName'], item_rm['unitName'], ))
        else:
            logger.error(
                '{}-{}-{} 房号列表响应解密出错'.format(item_rm['projectName'], item_rm['blockName'], item_rm['unitName'], ))

    def parse_room_detail(self, response):
        """
        获取房号详情
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        item_rm['roomId'] = response.xpath("//span[@id='HouseID']/text()").extract_first()
        item_rm['unitNo'] = response.xpath("//span[@id='SUBROOM']/text()").extract_first()
        item_rm['innerArea'] = response.xpath("//span[@id='AREA_INSIDE']/text()").extract_first()
        item_rm['apportionmentArea'] = response.xpath("//span[@id='AREA_SHARE']/text()").extract_first()
        item_rm['buildingArea'] = response.xpath("//span[@id='AREA_ROTAL']/text()").extract_first()
        yield item_rm

    @staticmethod
    def _deal_item_li(_li):
        """
        将列表字段转换为字符串
        :param _li:
        :return:
        """
        try:
            ret = ''.join(_li).strip()
            assert ret and ret != '0'
        except:
            return
        else:
            return ret

    def _get_room_li_url(self, _url):
        """
        获取房号列表url
        :param _url:
        :return:
        """
        try:
            params = _url.split('?')[1]
            ret = self.room_li_temp.format(params, random.random())
        except:
            return
        else:
            return ret

    @staticmethod
    def _deal_room_li_resp(resp):
        """
        处理房号列表加密响应
        :param resp:
        :return:
        """
        regex = re.compile(r'\+')
        try:
            ret = regex.sub(' ', urllib.parse.unquote(resp.body.decode()))
        except:
            return
        else:
            return ret
